{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"bert_lstm_crf.ipynb(mask矩阵和mask_label修改)","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyPrGYE2PaA2xqkiYetSRqEi"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"6dee61f7d28b4982b8694a041590b439":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_b74e8d919a054712a0d8fbc20c8ffe63","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_09dc3c4051e542818b01b9c7bc49a3b2","IPY_MODEL_365c2e07eed341a5a0c3a52925b673e2","IPY_MODEL_91e4b41be2d7491e96224e06516fa2e2"]}},"b74e8d919a054712a0d8fbc20c8ffe63":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"09dc3c4051e542818b01b9c7bc49a3b2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_2f5d2ed3e248424cb3a05d98f915d234","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_93d29b85383144239a688ee1fb971470"}},"365c2e07eed341a5a0c3a52925b673e2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_776c6e66a45e46f8809b7ce75f061542","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":11,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":11,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_7fd6780bf68f4207980e5e7a2560cfa9"}},"91e4b41be2d7491e96224e06516fa2e2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d178103d9ce14ffd8725656089b4a416","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 11/11 [00:06&lt;00:00,  1.86ba/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f48e1ac6dc384277b8d63340a824016a"}},"2f5d2ed3e248424cb3a05d98f915d234":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"93d29b85383144239a688ee1fb971470":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"776c6e66a45e46f8809b7ce75f061542":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"7fd6780bf68f4207980e5e7a2560cfa9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d178103d9ce14ffd8725656089b4a416":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f48e1ac6dc384277b8d63340a824016a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7c5b9d06bec144f4a057225dc4597248":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_b1b9ea9d49e94f4f988f22212155f7dc","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a66985c0276040daaee7361a18880c10","IPY_MODEL_ab1342ea0665458fae8235d5da709cf8","IPY_MODEL_3a7b9b330316441fb2196078d7b75bb3"]}},"b1b9ea9d49e94f4f988f22212155f7dc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a66985c0276040daaee7361a18880c10":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_edb8a6cf00ad4eaba9298b2dd4638577","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3b590a4505524a1a96e6734b4edcf930"}},"ab1342ea0665458fae8235d5da709cf8":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_668e62dcfbf44e47b5c8a7dada12ae87","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":2,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_18fe2e307f1a41029d061939d670eada"}},"3a7b9b330316441fb2196078d7b75bb3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f3421eab353046f2aa1e832855c3a76f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2/2 [00:00&lt;00:00,  2.27ba/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1aef072d911a48cfbde599276b5252d0"}},"edb8a6cf00ad4eaba9298b2dd4638577":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"3b590a4505524a1a96e6734b4edcf930":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"668e62dcfbf44e47b5c8a7dada12ae87":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"18fe2e307f1a41029d061939d670eada":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f3421eab353046f2aa1e832855c3a76f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1aef072d911a48cfbde599276b5252d0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"73b22f3ec33945bb9c133c9a3332c0cd":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_bbef918fb5204cc380416fdee0e3ab3a","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_e5b6e91ed38e46aca0cf211c3172c399","IPY_MODEL_e5438f8594e94828ba7bfb0b9e619a2b","IPY_MODEL_2ee483f8c21e4b109f2aa6762dca903a"]}},"bbef918fb5204cc380416fdee0e3ab3a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e5b6e91ed38e46aca0cf211c3172c399":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c0ee9e9993b94080bade76e7cb7e6199","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_dcc19cd55473421ca68e1181b4afb677"}},"e5438f8594e94828ba7bfb0b9e619a2b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_5d3c5558993049958be9db9d9d20fc79","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":2,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_566ee2685f014d3f82f886d68be09de3"}},"2ee483f8c21e4b109f2aa6762dca903a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d923e61403124b689c2ced28faa1a5fe","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2/2 [00:01&lt;00:00,  2.09ba/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_9b914006578d48f297c9064c4cfff6bb"}},"c0ee9e9993b94080bade76e7cb7e6199":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"dcc19cd55473421ca68e1181b4afb677":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5d3c5558993049958be9db9d9d20fc79":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"566ee2685f014d3f82f886d68be09de3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d923e61403124b689c2ced28faa1a5fe":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"9b914006578d48f297c9064c4cfff6bb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c4f82ea893f54d70bb6fc4d314a8dfd3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9f60ee3d383c4861bf6018b4fe93c29f","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_b750eb87e9b44002926403380ffdac99","IPY_MODEL_904d23bb1cef4383a804cf10e3552333","IPY_MODEL_cac4e6a63bda4d4a9984295f9c9dd2e8"]}},"9f60ee3d383c4861bf6018b4fe93c29f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b750eb87e9b44002926403380ffdac99":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9b644b14426547e1890b01431801a43a","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f3fe04a0eb1947dbb3a7df98caadd07a"}},"904d23bb1cef4383a804cf10e3552333":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_9765fcbba336499c9e2a6137fc9538a7","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"","max":6720,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":6720,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_948a231aa92042ee9d7cfb39fc1eea9b"}},"cac4e6a63bda4d4a9984295f9c9dd2e8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9b56c729183249099712a9b3b2747ba8","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 6720/6720 [2:21:15&lt;00:00,  1.17s/it]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f31febbf6d06451e9f118db012340997"}},"9b644b14426547e1890b01431801a43a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f3fe04a0eb1947dbb3a7df98caadd07a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9765fcbba336499c9e2a6137fc9538a7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"948a231aa92042ee9d7cfb39fc1eea9b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9b56c729183249099712a9b3b2747ba8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"f31febbf6d06451e9f118db012340997":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hw0bsFiqWXy6","executionInfo":{"status":"ok","timestamp":1640198620478,"user_tz":-480,"elapsed":3746,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"7fca76f0-32f6-4db1-9d28-928727cebd1b"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"code","metadata":{"id":"e01TmJXPXERL","executionInfo":{"status":"ok","timestamp":1640198620480,"user_tz":-480,"elapsed":8,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["import os\n","os.chdir('/content/drive/MyDrive/chinese task/CLUENER2020')"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"zalA7BN8XOEo","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1640198624776,"user_tz":-480,"elapsed":4303,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"194dc2af-3d2d-4722-da64-59387ab4143e"},"source":["#安装\n","!pip install transformers datasets pytorch-crf seqeval"],"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.14.1)\n","Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.17.0)\n","Requirement already satisfied: pytorch-crf in /usr/local/lib/python3.7/dist-packages (0.7.2)\n","Requirement already satisfied: seqeval in /usr/local/lib/python3.7/dist-packages (1.2.2)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.46)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n","Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.2.1)\n","Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n","Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n","Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n","Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n","Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n","Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2021.11.1)\n","Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.1)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval) (1.0.1)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.1.0)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.0.0)\n","Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.7.2)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (5.2.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n","Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n","Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n","Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n"]}]},{"cell_type":"code","metadata":{"id":"0a0Y29QSXciS","executionInfo":{"status":"ok","timestamp":1640198627812,"user_tz":-480,"elapsed":3043,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["import os\n","import json\n","import logging\n","import numpy as np\n","import pandas as pd\n","import config\n","\n","from sklearn.model_selection import train_test_split\n","from torch.utils.data import DataLoader"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9_0f6C8cI-uu","executionInfo":{"status":"ok","timestamp":1640198627813,"user_tz":-480,"elapsed":26,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"2137982c-f36a-4cb4-d515-16f973ecdf03"},"source":["#加载处理完的npz数据集\n","#不加allow_pickle=True会报错Object arrays cannot be loaded when allow_pickle=False，numpy新版本中默认为False。\n","train_data=np.load('./data/train.npz',allow_pickle=True)\n","val_data=np.load('./data/dev.npz',allow_pickle=True)\n","test_data=np.load('./data/test.npz',allow_pickle=True)\n","\n","test_data.files"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['words', 'labels']"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"4DGMH-td8_Sn","executionInfo":{"status":"ok","timestamp":1640198628465,"user_tz":-480,"elapsed":660,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"colab":{"base_uri":"https://localhost:8080/","height":423},"outputId":"4452c382-e089-4802-e754-5d9c5f5fe227"},"source":["#转换为dataframe格式\n","import pandas as pd\n","#补个随机frac\n","train_df=pd.concat([pd.DataFrame(train_data['words'],columns=['words']),\n","          pd.DataFrame(train_data['labels'],columns=['labels'])],axis=1).sample(frac=1.0)\n","#测试集和验证集不需要shuffle\n","val_df=pd.concat([pd.DataFrame(val_data['words'],columns=['words']),\n","          pd.DataFrame(val_data['labels'],columns=['labels'])],axis=1)#后面要进行预测对比标签，不宜shuffle\n","\n","test_df=pd.concat([pd.DataFrame(test_data['words'],columns=['words']),\n","          pd.DataFrame(test_data['labels'],columns=['labels'])],axis=1)\n","#小样本测试\n","#train_df=train_df.iloc[:1000]\n","#val_df=val_df.iloc[:500]\n","\n","\n","#将训练验证集的BIOS标签转换为数字索引，此时word和labels已经对齐了\n","def trans(labels):\n","  labels=list(labels)\n","  nums=[]\n","  for label in labels:\n","    nums.append(config.label2id[label])\n","  return nums\n","    \n","train_df['labels']=train_df['labels'].map(lambda x: trans(x))\n","val_df['labels']=val_df['labels'].map(lambda x: trans(x))\n","\n","test_df['labels']=test_df['labels'].map(lambda x: trans(x))\n","val_df"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","  <div id=\"df-9d888820-e48f-440d-9e59-de7bfd5fe6e9\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>words</th>\n","      <th>labels</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>[彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...</td>\n","      <td>[7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>[温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...</td>\n","      <td>[7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>[突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...</td>\n","      <td>[4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, 17, ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>[郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>[我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...</td>\n","      <td>[0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1338</th>\n","      <td>[在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>1339</th>\n","      <td>[姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...</td>\n","      <td>[6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n","    </tr>\n","    <tr>\n","      <th>1340</th>\n","      <td>[目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...</td>\n","      <td>[0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>1341</th>\n","      <td>[也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>1342</th>\n","      <td>[另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, 12, ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1343 rows × 2 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9d888820-e48f-440d-9e59-de7bfd5fe6e9')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-9d888820-e48f-440d-9e59-de7bfd5fe6e9 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-9d888820-e48f-440d-9e59-de7bfd5fe6e9');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "],"text/plain":["                                                  words                                             labels\n","0     [彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...  [7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...\n","1     [温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...  [7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...\n","2     [突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...  [4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, 17, ...\n","3     [郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...  [0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...\n","4     [我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...  [0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0...\n","...                                                 ...                                                ...\n","1338  [在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n","1339  [姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...  [6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...\n","1340  [目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...  [0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...\n","1341  [也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...  [0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0, 0, ...\n","1342  [另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...  [0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, 12, ...\n","\n","[1343 rows x 2 columns]"]},"metadata":{},"execution_count":6}]},{"cell_type":"markdown","metadata":{"id":"KzH2PsAaQY_s"},"source":["## pandas数据装入datasets进行解码，之后方便直接pad labels。\n","## labels在后面装入dataloader的时候，不处理的话长度不一致，处理的话整理函数太麻烦。"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":130,"referenced_widgets":["6dee61f7d28b4982b8694a041590b439","b74e8d919a054712a0d8fbc20c8ffe63","09dc3c4051e542818b01b9c7bc49a3b2","365c2e07eed341a5a0c3a52925b673e2","91e4b41be2d7491e96224e06516fa2e2","2f5d2ed3e248424cb3a05d98f915d234","93d29b85383144239a688ee1fb971470","776c6e66a45e46f8809b7ce75f061542","7fd6780bf68f4207980e5e7a2560cfa9","d178103d9ce14ffd8725656089b4a416","f48e1ac6dc384277b8d63340a824016a","7c5b9d06bec144f4a057225dc4597248","b1b9ea9d49e94f4f988f22212155f7dc","a66985c0276040daaee7361a18880c10","ab1342ea0665458fae8235d5da709cf8","3a7b9b330316441fb2196078d7b75bb3","edb8a6cf00ad4eaba9298b2dd4638577","3b590a4505524a1a96e6734b4edcf930","668e62dcfbf44e47b5c8a7dada12ae87","18fe2e307f1a41029d061939d670eada","f3421eab353046f2aa1e832855c3a76f","1aef072d911a48cfbde599276b5252d0","73b22f3ec33945bb9c133c9a3332c0cd","bbef918fb5204cc380416fdee0e3ab3a","e5b6e91ed38e46aca0cf211c3172c399","e5438f8594e94828ba7bfb0b9e619a2b","2ee483f8c21e4b109f2aa6762dca903a","c0ee9e9993b94080bade76e7cb7e6199","dcc19cd55473421ca68e1181b4afb677","5d3c5558993049958be9db9d9d20fc79","566ee2685f014d3f82f886d68be09de3","d923e61403124b689c2ced28faa1a5fe","9b914006578d48f297c9064c4cfff6bb"]},"id":"cbs_SgaUP19o","executionInfo":{"status":"ok","timestamp":1640198651634,"user_tz":-480,"elapsed":23177,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"788317c0-2b29-4c62-d73d-79be428a2db3"},"source":["from datasets import Dataset\n","from transformers import AutoTokenizer\n","#这里一定要选AutoTokenizer，如果是BertTokenizer，会提示bertbase没有word_ids方法。结果没用到\n","trains_ds=Dataset.from_pandas(train_df)\n","val_ds=Dataset.from_pandas(val_df)\n","test_ds=Dataset.from_pandas(test_df)\n","\n","tokenizer=AutoTokenizer.from_pretrained(config.roberta_model,do_lower_case=True)\n","\n","#tokenized_inputs=tokenizer(trains_ds[\"words\"],padding=True,truncation=True,is_split_into_words=True)为啥这种是错的\n","tokenized_trains_ds=trains_ds.map(lambda examples:tokenizer(examples['words'],is_split_into_words=True,truncation=True,padding=True),batched=True)\n","tokenized_val_ds=val_ds.map(lambda examples:tokenizer(examples['words'],is_split_into_words=True,truncation=True,padding=True),batched=True)\n","tokenized_test_ds=test_ds.map(lambda examples:tokenizer(examples['words'],is_split_into_words=True,truncation=True,padding=True),batched=True)"],"execution_count":7,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6dee61f7d28b4982b8694a041590b439","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/11 [00:00<?, ?ba/s]"]},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7c5b9d06bec144f4a057225dc4597248","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/2 [00:00<?, ?ba/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"73b22f3ec33945bb9c133c9a3332c0cd","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/2 [00:00<?, ?ba/s]"]},"metadata":{}}]},{"cell_type":"code","metadata":{"id":"A2N5xYkxn8qs","colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"status":"ok","timestamp":1640198657537,"user_tz":-480,"elapsed":5915,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"50e4dfc9-85c2-4e9f-bbac-f9fac3163933"},"source":["#在编码之后的datasets里面操作，得到的结果无法写入datasets，所以只好写到pandas文件里。\n","#将labels填充到和input_ids一样长（最长句子52，所以其实全部都填充到52）\n","\n","def padding(data,pad):\n","  pad_labels=[]\n","  for ds in data:\n","    labels=ds['labels'] \n","    mask=ds['attention_mask']\n","    label_ids=[pad]\n","\n","    pad_length=len(mask)\n","    label_length=len(labels)\n","    \n","    label_ids=label_ids+labels+[pad]*(pad_length-label_length-1)\n","    pad_labels.append(label_ids)\n","  return pad_labels\n","#tokenized_trains_ds[\"pad_labels\"]=pad_labels# Column 2 named labels expected length 10748 but got length 1000\n","\"\"\"\n","train_df['mask_labels']=padding(tokenized_trains_ds,-100)\n","val_df['mask_labels']=padding(tokenized_val_ds,-100)\n","test_df['mask_labels']=padding(tokenized_test_ds,-100)\"\"\"\n","\n","train_df['pad_labels']=padding(tokenized_trains_ds,-1)\n","val_df['pad_labels']=padding(tokenized_val_ds,-1)\n","test_df['pad_labels']=padding(tokenized_test_ds,-1)\n","val_df"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","  <div id=\"df-5a91e445-286a-4f3f-b684-89084cb68d16\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>words</th>\n","      <th>labels</th>\n","      <th>pad_labels</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>[彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...</td>\n","      <td>[7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[-1, 7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>[温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...</td>\n","      <td>[7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n","      <td>[-1, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>[突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...</td>\n","      <td>[4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, 17, ...</td>\n","      <td>[-1, 4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>[郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[-1, 0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>[我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...</td>\n","      <td>[0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[-1, 0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1338</th>\n","      <td>[在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","      <td>[-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n","    </tr>\n","    <tr>\n","      <th>1339</th>\n","      <td>[姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...</td>\n","      <td>[6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n","      <td>[-1, 6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n","    </tr>\n","    <tr>\n","      <th>1340</th>\n","      <td>[目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...</td>\n","      <td>[0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...</td>\n","      <td>[-1, 0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 1...</td>\n","    </tr>\n","    <tr>\n","      <th>1341</th>\n","      <td>[也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0, 0, ...</td>\n","      <td>[-1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0,...</td>\n","    </tr>\n","    <tr>\n","      <th>1342</th>\n","      <td>[另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, 12, ...</td>\n","      <td>[-1, 0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1343 rows × 3 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5a91e445-286a-4f3f-b684-89084cb68d16')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-5a91e445-286a-4f3f-b684-89084cb68d16 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-5a91e445-286a-4f3f-b684-89084cb68d16');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "],"text/plain":["                                                  words  ...                                         pad_labels\n","0     [彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...  ...  [-1, 7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n","1     [温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...  ...  [-1, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...\n","2     [突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...  ...  [-1, 4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, ...\n","3     [郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...  ...  [-1, 0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, ...\n","4     [我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...  ...  [-1, 0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, ...\n","...                                                 ...  ...                                                ...\n","1338  [在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...  ...  [-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...\n","1339  [姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...  ...  [-1, 6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...\n","1340  [目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...  ...  [-1, 0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 1...\n","1341  [也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...  ...  [-1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0,...\n","1342  [另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...  ...  [-1, 0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, ...\n","\n","[1343 rows x 3 columns]"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"eUjuf-chiW8I","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1640198668672,"user_tz":-480,"elapsed":11139,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"5f9c7d9a-42d8-41f6-907d-54999af5b65c"},"source":["batch_size=16\n","\n","#划分训练验证集\n","from sklearn.model_selection import train_test_split\n","from datasets import Dataset\n","from torch.nn.utils.rnn import pad_sequence\n","\n","\n","train_data,train_label,val_data,val_label=train_df['words'].iloc[:],train_df['pad_labels'].iloc[:],val_df['words'].iloc[:],val_df['pad_labels'].iloc[:]\n","\n","test_data,test_label=(test_df['words'].iloc[:],test_df['pad_labels'].iloc[:])\n","\n","#stratify=train_df['label'].iloc[:]报错:The least populated class in y has only 1 member,which is too few.\n","#The minimum number of groups for any class cannot be less than 2.估计是样本太少，分层抽取不可行。\n","\n","#数据预处理\n","\n","tokenizer=AutoTokenizer.from_pretrained(config.roberta_model,do_lower_case=True)\n","train_encoding=tokenizer(list(train_data),is_split_into_words=True,truncation=True,padding=True,return_tensors='pt')#训练集中划分的训练集\n","val_encoding=tokenizer(list(val_data),is_split_into_words=True,truncation=True,padding=True,return_tensors='pt')#训练集中划分的验证集\n","test_encoding=tokenizer(list(test_data),is_split_into_words=True,truncation=True,padding=True,return_tensors='pt')#测试集"],"execution_count":9,"outputs":[{"output_type":"stream","name":"stderr","text":["Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"]}]},{"cell_type":"code","metadata":{"id":"4O2SbmyDskK6","executionInfo":{"status":"ok","timestamp":1640198668673,"user_tz":-480,"elapsed":14,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["#加载到datalodar并预处理\n","#数据集读取\n","\n","from torch.utils.data import Dataset, DataLoader,TensorDataset\n","import torch\n","class XFeiDataset(Dataset):\n","  def __init__(self,encodings,pad_labels):\n","    self.encodings=encodings\n","    self.pad_labels=pad_labels\n","  \n","  # 读取单个样本\n","  def __getitem__(self,idx):\n","    item={key:torch.tensor(val[idx]) for key,val in self.encodings.items()}\n","    item['pad_labels']=torch.tensor((self.pad_labels[idx]))\n","    item['mask']=(item['pad_labels']!=-1)\n","    return item\n","  \n","  def __len__(self):\n","    return len(self.pad_labels)\n","\n","#def collate_fn\n","\n","train_dataset=XFeiDataset(train_encoding,list(train_label))\n","val_dataset=XFeiDataset(val_encoding,list(val_label))\n","test_dataset=XFeiDataset(test_encoding,list(test_label))\n","\n","\n","from torch.utils.data import Dataset,DataLoader,TensorDataset\n","\n","train_loader=DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\n","val_loader=DataLoader(val_dataset,batch_size=batch_size,shuffle=False)\n","test_loader=DataLoader(test_dataset,batch_size=batch_size,shuffle=False)#test数据不能shuffle啊，真坑死我了"],"execution_count":10,"outputs":[]},{"cell_type":"code","source":["#for i in val_loader:\n","  #print(i)#输出5元组，input三兄弟和pad_labels和mask矩阵"],"metadata":{"id":"PWyyldK_j4NA","executionInfo":{"status":"ok","timestamp":1640198668673,"user_tz":-480,"elapsed":12,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"vZh3myklQsHo","executionInfo":{"status":"ok","timestamp":1640198668674,"user_tz":-480,"elapsed":12,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["from transformers import BertModel\n","from torch.nn.utils.rnn import pad_sequence\n","#初始化bert模型\n","from transformers import BertConfig\n","import torch.nn as nn\n","from torch.nn import LSTM\n","from torch.nn import functional as F \n","from torchcrf import CRF\n","\n","num_labels=31\n","dropout=0.1\n","#取0.1时，epoch=1，precision 0.68|recall 0.72|f1 0.70|acc 0.93\n","#选0.2训练loss更大。epoch1时precision 0.50|recall 0.60|f1 0.54|acc 0.90\n","\n","class Bert_LSTM(nn.Module):\n","  def __init__(self):\n","    super(Bert_LSTM,self).__init__()\n","    self.num_labels=num_labels\n","    self.dropout=nn.Dropout(dropout)\n","    self.bert=BertModel.from_pretrained(config.roberta_model)\n","    for param in self.bert.parameters():\n","      param.requires_grad=True\n","    self.classifier=nn.Linear(1024,self.num_labels)\n","    self.crf=CRF(num_labels,batch_first=True)\n","    from torch.nn import functional as F\n","\n","    self.bilstm=nn.LSTM(\n","        input_size=1024, \n","        hidden_size=512, \n","        batch_first=True,\n","        num_layers=2,\n","        dropout=0.5,  \n","        bidirectional=True)\n","\n","  def forward(self,batch_seqs,batch_seq_masks,batch_seq_segments,pad_labels,mask):\n","\n","    output=self.bert(input_ids=batch_seqs,attention_mask=batch_seq_masks,token_type_ids=batch_seq_segments)\n","    #pooler_output=output.pooler_output\n","    last_hidden_state=output.last_hidden_state\n","    last_hidden_state=self.dropout(last_hidden_state)\n","    #只有这种写法不会报错，如果是sequence_output,pooler_output=self.bert(**kwags)这种，sequence_output会报错str没有xxx属性。\n","    #貌似是bert输出有很多，直接用output.last_hidden_state来调用结果（估计是版本问题，坑），关键是输出要打印出来\n","    \n","\n","    lstm_output,(hn,cn)=self.bilstm(last_hidden_state)\n","    #output为输出序列的隐藏层，hn为最后一个时刻的隐藏层，cn为最后一个时刻的隐藏细胞\n","    lstm_output=self.dropout(lstm_output)\n","\n","    # 得到判别值\n","    logits=self.classifier(lstm_output)\n","    logits,pad_labels,mask=logits[:,1:,:],pad_labels[:,1:],mask[:,1:]#首个值不能是false，否则crf报错\n","    loss=self.crf(logits,pad_labels,mask)*(-1)\n","    \n","    return logits,loss"],"execution_count":12,"outputs":[]},{"cell_type":"code","metadata":{"id":"RcKd7JbCIu-b","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1640198684752,"user_tz":-480,"elapsed":16090,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"37a0da0c-9303-47b5-f17e-91ccf1a3ede2"},"source":["#加载模型\n","model=Bert_LSTM()\n","#model.load_state_dict(torch.load(\"/content/drive/MyDrive/chinese task/CLUENER2020/model/bert_lstm_crf_model\"))\n","device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","model.to(device)"],"execution_count":13,"outputs":[{"output_type":"stream","name":"stderr","text":["Some weights of the model checkpoint at hfl/chinese-roberta-wwm-ext-large were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight']\n","- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"]},{"output_type":"execute_result","data":{"text/plain":["Bert_LSTM(\n","  (dropout): Dropout(p=0.1, inplace=False)\n","  (bert): BertModel(\n","    (embeddings): BertEmbeddings(\n","      (word_embeddings): Embedding(21128, 1024, padding_idx=1)\n","      (position_embeddings): Embedding(512, 1024)\n","      (token_type_embeddings): Embedding(2, 1024)\n","      (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (encoder): BertEncoder(\n","      (layer): ModuleList(\n","        (0): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (1): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (2): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (3): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (4): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (5): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (6): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (7): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (8): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (9): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (10): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (11): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (12): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (13): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (14): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (15): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (16): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (17): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (18): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (19): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (20): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (21): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (22): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (23): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=1024, out_features=1024, bias=True)\n","              (key): Linear(in_features=1024, out_features=1024, bias=True)\n","              (value): Linear(in_features=1024, out_features=1024, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","              (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n","            (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","      )\n","    )\n","    (pooler): BertPooler(\n","      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n","      (activation): Tanh()\n","    )\n","  )\n","  (classifier): Linear(in_features=1024, out_features=31, bias=True)\n","  (crf): CRF(num_tags=31)\n","  (bilstm): LSTM(1024, 512, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",")"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"yf0ux4z6V7vk","executionInfo":{"status":"ok","timestamp":1640198684753,"user_tz":-480,"elapsed":21,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["#定义优化器\n","epoch=10\n","lr=3e-5\n","\n","from transformers import AdamW,get_scheduler\n","\n","train_steps_per_epoch=len(train_loader)\n","num_training_steps=train_steps_per_epoch*epoch\n","\n","#定义各模块参数\n","bert_parameters=list(model.bert.named_parameters())\n","lstm_parameters=list(model.bilstm.named_parameters())\n","classifier_parameters=list(model.classifier.named_parameters())\n","no_decay=['bias','LayerNorm.weight']\n","\n","#bert模型、lstm模型、nn.linear的学习率分离，后两个是bert的3倍\n","optimizer_grouped_parameters=[\n","    {'params':[p for n,p in bert_parameters if not any(nd in n for nd in no_decay)],\n","      'lr':lr,'weight_decay':0.01},\n","    {'params':[p for n,p in bert_parameters if any(nd in n for nd in no_decay)],\n","      'lr':lr,'weight_decay':0.0},\n","    {'params':[p for n,p in lstm_parameters if not any(nd in n for nd in no_decay)],\n","      'lr':lr*3,'weight_decay':0.01},\n","    {'params':[p for n,p in lstm_parameters if any(nd in n for nd in no_decay)],\n","      'lr':lr*3,'weight_decay': 0.0},\n","    {'params':[p for n,p in classifier_parameters if not any(nd in n for nd in no_decay)],\n","      'lr':lr*3,'weight_decay':0.01},\n","    {'params':[p for n,p in classifier_parameters if any(nd in n for nd in no_decay)],\n","      'lr':lr*3,'weight_decay':0.0}]\n","\n","optimizer=AdamW(optimizer_grouped_parameters,lr=lr,eps=1e-8)\n","#使用线性衰减学习率\n","lr_scheduler=get_scheduler(\n","    \"linear\",\n","    optimizer=optimizer,\n","    num_warmup_steps=0,\n","    num_training_steps=num_training_steps)"],"execution_count":14,"outputs":[]},{"cell_type":"code","source":["#将crf.decode预测值用0进行pad之后转为tensor\n","def pad_result(data,pad_labels):\n","  pad_pred=[]\n","  max_len=pad_labels.shape[1]\n","  for pred in data:\n","    pad_length=max_len-len(pred)\n","\n","    label_ids=pred+[0]*(pad_length)\n","    pad_pred.append(label_ids)\n","    \n","  return pad_pred\n","\n","#pred=torch.tensor(pred)必须在函数外，否则报错tensor没有append\n","\n","#编写评价方法\n","from datasets import load_metric\n","metric=load_metric(\"seqeval\")\n","import numpy as np\n","\n","label_list= [label for label,id in list(config.label2id.items())]\n","\n","def compute_metrics(y_pred,y_true):\n","  predictions,labels=y_pred,y_true\n","\n","  # 掉特殊字符处的值，不作比较。\n","  true_predictions=[\n","    [label_list[p] for (p,l) in zip(prediction, label) if l !=-1]\n","    for prediction,label in zip(predictions, labels)\n","  ]\n","  true_labels=[\n","    [label_list[l] for (p,l) in zip(prediction, label) if l !=-1]\n","    for prediction,label in zip(predictions, labels)\n","  ]\n","\n","  results = metric.compute(predictions=true_predictions,references=true_labels)\n","  return results\n"],"metadata":{"id":"jqQIhRuAjBN3","executionInfo":{"status":"ok","timestamp":1640198685725,"user_tz":-480,"elapsed":993,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"id":"LFe1R8aMcCL7","colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["c4f82ea893f54d70bb6fc4d314a8dfd3","9f60ee3d383c4861bf6018b4fe93c29f","b750eb87e9b44002926403380ffdac99","904d23bb1cef4383a804cf10e3552333","cac4e6a63bda4d4a9984295f9c9dd2e8","9b644b14426547e1890b01431801a43a","f3fe04a0eb1947dbb3a7df98caadd07a","9765fcbba336499c9e2a6137fc9538a7","948a231aa92042ee9d7cfb39fc1eea9b","9b56c729183249099712a9b3b2747ba8","f31febbf6d06451e9f118db012340997"]},"executionInfo":{"status":"ok","timestamp":1640198687322,"user_tz":-480,"elapsed":9,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"47d44581-e0ee-4733-c68a-b70340904b20"},"source":["#编写训练和验证循环\n","import time\n","import numpy as np\n","from sklearn.metrics import f1_score,precision_score,recall_score,accuracy_score\n","from torch.nn import functional as F\n","from torchcrf import CRF\n","#加载进度条\n","from tqdm.auto import tqdm\n","\n","num_training_steps=train_steps_per_epoch*epoch\n","\n","progress_bar=tqdm(range(num_training_steps))\n","\n","def train_and_eval(epoch):\n","  for i in range(epoch):\n","    \"\"\"训练模型\"\"\"\n","    start=time.time()\n","    model.train()\n","    print(\"***** Running training epoch {} *****\".format(i+1))\n","    train_loss_sum=0.0\n","    for idx,batch in enumerate(train_loader):\n","      input_ids=batch['input_ids'].to(device)\n","      attention_mask=batch['attention_mask'].to(device)\n","      token_type_ids=batch['token_type_ids'].to(device)\n","      pad_labels=batch['pad_labels'].to(device)\n","      mask=batch['mask'].to(device)\n","\n","\n","      #计算输出和loss\n","      logits,loss=model(input_ids,attention_mask,token_type_ids,pad_labels,mask)\n","      loss.backward()\n","\n","      optimizer.step()\n","      lr_scheduler.step()\n","      optimizer.zero_grad()  \n","      progress_bar.update(1)\n","\n","      train_loss_sum+=loss.item()\n","      if (idx+1)%(len(train_loader)//5)==0: # 只打印五次结果\n","        print(\"Epoch {:03d} | Step {:04d}/{:04d} | Loss {:.4f} | Time {:.4f} | Learning rate = {} \\n\".format(\n","                  i+1,idx+1,len(train_loader),train_loss_sum/(idx+1),time.time()-start,optimizer.state_dict()['param_groups'][0]['lr']))\n","      \n","      #验证模型\n","    model.eval()\n","    y_pred,y_true=[],[]\n","    best_f1,total_eval_loss=0,0\n","    total_eval_accuracy,total,acc=0,0,0\n","    \n","    for batch in val_loader:\n","      with torch.no_grad():#只有这一块是不需要求导的\n","      \n","        input_ids=batch['input_ids'].to(device)\n","        attention_mask=batch['attention_mask'].to(device)\n","        token_type_ids=batch['token_type_ids'].to(device)\n","        pad_labels=batch['pad_labels'].to(device)\n","        mask=batch['mask'].to(device)\n","        logits,loss=model(input_ids,attention_mask,token_type_ids,pad_labels,mask)#都是去掉首个-100的结果                      \n","           \n","      total_eval_loss+=loss.item()\n","      \n","\n","      pad_labels,mask=pad_labels[:,1:].to(device),mask[:,1:].to(device)\n","      pred=model.crf.decode(logits,mask)\n","      pred=torch.tensor(pad_result(pred,pad_labels)).to(device)#预测值经过pad_pred函数pad成batch_size*max_length-1，再转为tensor\n","      acc+=(pred[mask]==pad_labels[mask]).sum().item()#只计算没有mask的单词的准确率,mask在外面似乎accs0.93不准。\n","      total+=mask.sum().item()\n","      total_eval_accuracy=acc/total\n","\n","      y_pred.extend(pred.cpu().numpy().tolist())#将每个把batch结果依次加入总列表\n","      y_true.extend(pad_labels.cpu().numpy().tolist())\n","    results=compute_metrics(y_pred,y_true)\n","    f1=results[\"overall_f1\"]    \n","    \n","    if f1>best_f1:\n","      best_f1=f1\n","      torch.save(model.state_dict(),\"./bert_lstm_crf/blf_model\")\n","    \n","    print(\"precision {:.2f}|recall {:.2f}|f1 {:.4f}|acc {:.2f}\".format(results[\"overall_precision\"],results[\"overall_recall\"],results[\"overall_f1\"],results[\"overall_accuracy\"]))\n","    print(\"Average val loss:%.2f\"%(total_eval_loss),\"sklearn_acc:%.2f\"%(total_eval_accuracy))\n","    print(\"time costed={}s \\n\".format(round(time.time()-start,5)))\n","    print(\"-------------------------------\")"],"execution_count":16,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c4f82ea893f54d70bb6fc4d314a8dfd3","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/6720 [00:00<?, ?it/s]"]},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"Y_Ahn2d251Tt"},"source":["## 对比bert的token分类任务头"]},{"cell_type":"code","metadata":{"id":"sORb6WSDVejg","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"20624f11-e4b6-4e77-ed06-1bb2a8aef793","executionInfo":{"status":"ok","timestamp":1640207204108,"user_tz":-480,"elapsed":3197689,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"source":["train_and_eval(epoch)\n","\"\"\"\n","预测值不包括实际值会报错\n","UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\"\"\""],"execution_count":17,"outputs":[{"metadata":{"tags":null},"name":"stdout","output_type":"stream","text":["***** Running training epoch 1 *****\n"]},{"metadata":{"tags":null},"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  del sys.path[0]\n"]},{"output_type":"stream","name":"stdout","text":["Epoch 001 | Step 0134/0672 | Loss 850.4533 | Time 160.0524 | Learning rate = 2.9401785714285713e-05 \n","\n","Epoch 001 | Step 0268/0672 | Loss 755.6008 | Time 321.1079 | Learning rate = 2.880357142857143e-05 \n","\n","Epoch 001 | Step 0402/0672 | Loss 693.5408 | Time 482.2713 | Learning rate = 2.8205357142857143e-05 \n","\n","Epoch 001 | Step 0536/0672 | Loss 626.1115 | Time 643.2802 | Learning rate = 2.7607142857142855e-05 \n","\n","Epoch 001 | Step 0670/0672 | Loss 554.6280 | Time 804.1686 | Learning rate = 2.7008928571428574e-05 \n","\n","precision 0.55|recall 0.65|f1 0.59|acc 0.90\n","Average val loss:16898.68 sklearn_acc:0.90\n","time costed=847.54825s \n","\n","-------------------------------\n","***** Running training epoch 2 *****\n","Epoch 002 | Step 0134/0672 | Loss 189.6032 | Time 161.3824 | Learning rate = 2.6401785714285714e-05 \n","\n","Epoch 002 | Step 0268/0672 | Loss 174.3528 | Time 321.8987 | Learning rate = 2.580357142857143e-05 \n","\n","Epoch 002 | Step 0402/0672 | Loss 166.6937 | Time 482.3897 | Learning rate = 2.5205357142857145e-05 \n","\n","Epoch 002 | Step 0536/0672 | Loss 160.9221 | Time 643.1670 | Learning rate = 2.4607142857142857e-05 \n","\n","Epoch 002 | Step 0670/0672 | Loss 155.2287 | Time 803.8956 | Learning rate = 2.4008928571428572e-05 \n","\n","precision 0.70|recall 0.78|f1 0.73|acc 0.94\n","Average val loss:10865.32 sklearn_acc:0.94\n","time costed=846.85732s \n","\n","-------------------------------\n","***** Running training epoch 3 *****\n","Epoch 003 | Step 0134/0672 | Loss 104.7770 | Time 161.5656 | Learning rate = 2.3401785714285716e-05 \n","\n","Epoch 003 | Step 0268/0672 | Loss 104.1795 | Time 322.5888 | Learning rate = 2.2803571428571428e-05 \n","\n","Epoch 003 | Step 0402/0672 | Loss 102.1001 | Time 483.3302 | Learning rate = 2.2205357142857143e-05 \n","\n","Epoch 003 | Step 0536/0672 | Loss 100.6387 | Time 644.2039 | Learning rate = 2.1607142857142858e-05 \n","\n","Epoch 003 | Step 0670/0672 | Loss 100.2379 | Time 804.9622 | Learning rate = 2.100892857142857e-05 \n","\n","precision 0.71|recall 0.79|f1 0.75|acc 0.94\n","Average val loss:10455.55 sklearn_acc:0.94\n","time costed=847.52307s \n","\n","-------------------------------\n","***** Running training epoch 4 *****\n","Epoch 004 | Step 0134/0672 | Loss 65.6758 | Time 161.4387 | Learning rate = 2.0401785714285714e-05 \n","\n","Epoch 004 | Step 0268/0672 | Loss 70.1131 | Time 321.9805 | Learning rate = 1.980357142857143e-05 \n","\n","Epoch 004 | Step 0402/0672 | Loss 74.5657 | Time 482.5696 | Learning rate = 1.920535714285714e-05 \n","\n","Epoch 004 | Step 0536/0672 | Loss 77.0551 | Time 643.0697 | Learning rate = 1.860714285714286e-05 \n","\n","Epoch 004 | Step 0670/0672 | Loss 77.2134 | Time 803.5144 | Learning rate = 1.800892857142857e-05 \n","\n","precision 0.72|recall 0.79|f1 0.75|acc 0.94\n","Average val loss:10545.26 sklearn_acc:0.94\n","time costed=846.01025s \n","\n","-------------------------------\n","***** Running training epoch 5 *****\n","Epoch 005 | Step 0134/0672 | Loss 62.8429 | Time 161.6965 | Learning rate = 1.7401785714285716e-05 \n","\n","Epoch 005 | Step 0268/0672 | Loss 61.3453 | Time 322.8604 | Learning rate = 1.680357142857143e-05 \n","\n","Epoch 005 | Step 0402/0672 | Loss 59.3090 | Time 484.0892 | Learning rate = 1.6205357142857143e-05 \n","\n","Epoch 005 | Step 0536/0672 | Loss 58.5966 | Time 645.2598 | Learning rate = 1.5607142857142858e-05 \n","\n","Epoch 005 | Step 0670/0672 | Loss 57.8091 | Time 806.5770 | Learning rate = 1.5008928571428572e-05 \n","\n","precision 0.73|recall 0.81|f1 0.76|acc 0.94\n","Average val loss:11918.01 sklearn_acc:0.94\n","time costed=849.09279s \n","\n","-------------------------------\n","***** Running training epoch 6 *****\n","Epoch 006 | Step 0134/0672 | Loss 40.8853 | Time 161.7673 | Learning rate = 1.4401785714285716e-05 \n","\n","Epoch 006 | Step 0268/0672 | Loss 40.4048 | Time 322.7583 | Learning rate = 1.3803571428571427e-05 \n","\n","Epoch 006 | Step 0402/0672 | Loss 40.0783 | Time 483.8099 | Learning rate = 1.3205357142857143e-05 \n","\n","Epoch 006 | Step 0536/0672 | Loss 39.7375 | Time 644.7180 | Learning rate = 1.2607142857142858e-05 \n","\n","Epoch 006 | Step 0670/0672 | Loss 40.0972 | Time 805.9105 | Learning rate = 1.2008928571428573e-05 \n","\n","precision 0.74|recall 0.81|f1 0.78|acc 0.94\n","Average val loss:11576.13 sklearn_acc:0.94\n","time costed=848.65729s \n","\n","-------------------------------\n","***** Running training epoch 7 *****\n","Epoch 007 | Step 0134/0672 | Loss 28.1865 | Time 161.6997 | Learning rate = 1.1401785714285714e-05 \n","\n","Epoch 007 | Step 0268/0672 | Loss 29.4536 | Time 322.6295 | Learning rate = 1.0803571428571429e-05 \n","\n","Epoch 007 | Step 0402/0672 | Loss 29.7340 | Time 483.5822 | Learning rate = 1.0205357142857144e-05 \n","\n","Epoch 007 | Step 0536/0672 | Loss 30.0218 | Time 644.7393 | Learning rate = 9.607142857142856e-06 \n","\n","Epoch 007 | Step 0670/0672 | Loss 29.1320 | Time 805.7908 | Learning rate = 9.008928571428571e-06 \n","\n","precision 0.75|recall 0.80|f1 0.78|acc 0.94\n","Average val loss:13548.30 sklearn_acc:0.94\n","time costed=848.7225s \n","\n","-------------------------------\n","***** Running training epoch 8 *****\n","Epoch 008 | Step 0134/0672 | Loss 19.5674 | Time 162.0373 | Learning rate = 8.401785714285715e-06 \n","\n","Epoch 008 | Step 0268/0672 | Loss 20.5346 | Time 323.2465 | Learning rate = 7.803571428571429e-06 \n","\n","Epoch 008 | Step 0402/0672 | Loss 20.5707 | Time 484.4724 | Learning rate = 7.205357142857143e-06 \n","\n","Epoch 008 | Step 0536/0672 | Loss 21.3377 | Time 645.7742 | Learning rate = 6.607142857142857e-06 \n","\n","Epoch 008 | Step 0670/0672 | Loss 21.3365 | Time 806.9678 | Learning rate = 6.008928571428572e-06 \n","\n","precision 0.75|recall 0.80|f1 0.78|acc 0.94\n","Average val loss:14625.68 sklearn_acc:0.94\n","time costed=850.00101s \n","\n","-------------------------------\n","***** Running training epoch 9 *****\n","Epoch 009 | Step 0134/0672 | Loss 15.8689 | Time 162.2782 | Learning rate = 5.4017857142857145e-06 \n","\n","Epoch 009 | Step 0268/0672 | Loss 15.7494 | Time 323.9999 | Learning rate = 4.803571428571428e-06 \n","\n","Epoch 009 | Step 0402/0672 | Loss 16.1067 | Time 485.5929 | Learning rate = 4.205357142857143e-06 \n","\n","Epoch 009 | Step 0536/0672 | Loss 15.9825 | Time 647.0968 | Learning rate = 3.6071428571428573e-06 \n","\n","Epoch 009 | Step 0670/0672 | Loss 15.6784 | Time 808.7016 | Learning rate = 3.0089285714285717e-06 \n","\n","precision 0.74|recall 0.81|f1 0.77|acc 0.94\n","Average val loss:15238.61 sklearn_acc:0.94\n","time costed=851.86068s \n","\n","-------------------------------\n","***** Running training epoch 10 *****\n","Epoch 010 | Step 0134/0672 | Loss 12.5352 | Time 162.6241 | Learning rate = 2.401785714285714e-06 \n","\n","Epoch 010 | Step 0268/0672 | Loss 12.0666 | Time 327.8486 | Learning rate = 1.8035714285714286e-06 \n","\n","Epoch 010 | Step 0402/0672 | Loss 12.0815 | Time 492.9378 | Learning rate = 1.205357142857143e-06 \n","\n","Epoch 010 | Step 0536/0672 | Loss 12.2230 | Time 658.1718 | Learning rate = 6.071428571428572e-07 \n","\n","Epoch 010 | Step 0670/0672 | Loss 11.8025 | Time 823.4604 | Learning rate = 8.92857142857143e-09 \n","\n","precision 0.75|recall 0.80|f1 0.77|acc 0.94\n","Average val loss:15777.25 sklearn_acc:0.94\n","time costed=868.35004s \n","\n","-------------------------------\n"]},{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'\\n预测值不包括实际值会报错\\nUndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\\n  _warn_prf(average, modifier, msg_start, len(result))'"]},"metadata":{},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"EvTMpEjLKpZx"},"source":["#torch.save(model.state_dict(),\"./bert_lstm_crf/finall_blf_model\")\n","model.load_state_dict(torch.load(\"./bert_lstm_crf/blf_model\"))\n","model.to(device)"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#编写predict函数\n","def predict(model,data_loader):#参数名为data时加载训练好的模型来预测报错，原模型不报错\n","  model.eval()\n","  y_pred,y_true,predictions= [],[],[]#pad后的预测结果、真实标签和没有pad的标签list\n","  for batch in data_loader:\n","    with torch.no_grad():#只有这一块是不需要求导的\n","      \n","      input_ids=batch['input_ids'].to(device)\n","      attention_mask=batch['attention_mask'].to(device)\n","      token_type_ids=batch['token_type_ids'].to(device)\n","      pad_labels=batch['pad_labels'].to(device)#用1填充的标签，用来计算logits和loss\n","      mask=batch['mask'].to(device)#mask矩阵，用来计算logits和loss、crf解码结果\n","      logits,loss=model(input_ids,attention_mask,token_type_ids,pad_labels,mask)                      \n","    \n","    pad_labels,mask=pad_labels[:,1:].to(device),mask[:,1:].to(device)\n","    prediction=model.crf.decode(logits,mask)#解码后得出真实tokens的预测结果list\n","\n","    #预测值只是列表，pad之后加入总的预测列表，用于评价compute_metrics\n","    pad_pred=torch.tensor(pad_result(prediction,pad_labels)).to(device)\n","    y_pred.extend(pad_pred.cpu().numpy().tolist())\n","    y_true.extend(pad_labels.cpu().numpy().tolist())#pad_labels为真实标签\n","\n","    predictions.extend(prediction)#这个是不用pad的预测结果，用于最终提交结果\n","      \n","  return y_pred,y_true,predictions"],"metadata":{"id":"dZg4Xb8Xql6B","executionInfo":{"status":"ok","timestamp":1640208276564,"user_tz":-480,"elapsed":522,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["#预测验证集结果，查看各个tokens类别的指标\n","y_pred,y_true,predictions=predict(model,val_loader)\n","results=compute_metrics(y_pred,y_true)\n","#将结果排序查看\n","result_df=pd.DataFrame(results)\n","result_df.stack().unstack(0).sort_values(by=['f1'])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":543},"id":"1VM6ATdorElv","executionInfo":{"status":"ok","timestamp":1640208315181,"user_tz":-480,"elapsed":36224,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"542780b4-cdcc-4661-c8f8-a8fdefd62f83"},"execution_count":28,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  del sys.path[0]\n"]},{"output_type":"execute_result","data":{"text/html":["\n","  <div id=\"df-696db27c-5ac8-495c-aa6f-406fba9a1c0b\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>precision</th>\n","      <th>recall</th>\n","      <th>f1</th>\n","      <th>number</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>address</th>\n","      <td>0.555288</td>\n","      <td>0.619303</td>\n","      <td>0.585551</td>\n","      <td>373.000000</td>\n","    </tr>\n","    <tr>\n","      <th>scene</th>\n","      <td>0.697115</td>\n","      <td>0.693780</td>\n","      <td>0.695444</td>\n","      <td>209.000000</td>\n","    </tr>\n","    <tr>\n","      <th>overall_precision</th>\n","      <td>0.747256</td>\n","      <td>0.747256</td>\n","      <td>0.747256</td>\n","      <td>0.747256</td>\n","    </tr>\n","    <tr>\n","      <th>organization</th>\n","      <td>0.721805</td>\n","      <td>0.784741</td>\n","      <td>0.751958</td>\n","      <td>367.000000</td>\n","    </tr>\n","    <tr>\n","      <th>overall_f1</th>\n","      <td>0.771725</td>\n","      <td>0.771725</td>\n","      <td>0.771725</td>\n","      <td>0.771725</td>\n","    </tr>\n","    <tr>\n","      <th>government</th>\n","      <td>0.739130</td>\n","      <td>0.825911</td>\n","      <td>0.780115</td>\n","      <td>247.000000</td>\n","    </tr>\n","    <tr>\n","      <th>position</th>\n","      <td>0.774554</td>\n","      <td>0.801386</td>\n","      <td>0.787741</td>\n","      <td>433.000000</td>\n","    </tr>\n","    <tr>\n","      <th>book</th>\n","      <td>0.775000</td>\n","      <td>0.805195</td>\n","      <td>0.789809</td>\n","      <td>154.000000</td>\n","    </tr>\n","    <tr>\n","      <th>company</th>\n","      <td>0.759124</td>\n","      <td>0.825397</td>\n","      <td>0.790875</td>\n","      <td>378.000000</td>\n","    </tr>\n","    <tr>\n","      <th>overall_recall</th>\n","      <td>0.797852</td>\n","      <td>0.797852</td>\n","      <td>0.797852</td>\n","      <td>0.797852</td>\n","    </tr>\n","    <tr>\n","      <th>movie</th>\n","      <td>0.807947</td>\n","      <td>0.807947</td>\n","      <td>0.807947</td>\n","      <td>151.000000</td>\n","    </tr>\n","    <tr>\n","      <th>game</th>\n","      <td>0.791925</td>\n","      <td>0.864407</td>\n","      <td>0.826580</td>\n","      <td>295.000000</td>\n","    </tr>\n","    <tr>\n","      <th>name</th>\n","      <td>0.865031</td>\n","      <td>0.909677</td>\n","      <td>0.886792</td>\n","      <td>465.000000</td>\n","    </tr>\n","    <tr>\n","      <th>overall_accuracy</th>\n","      <td>0.938997</td>\n","      <td>0.938997</td>\n","      <td>0.938997</td>\n","      <td>0.938997</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-696db27c-5ac8-495c-aa6f-406fba9a1c0b')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-696db27c-5ac8-495c-aa6f-406fba9a1c0b button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-696db27c-5ac8-495c-aa6f-406fba9a1c0b');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "],"text/plain":["                   precision    recall        f1      number\n","address             0.555288  0.619303  0.585551  373.000000\n","scene               0.697115  0.693780  0.695444  209.000000\n","overall_precision   0.747256  0.747256  0.747256    0.747256\n","organization        0.721805  0.784741  0.751958  367.000000\n","overall_f1          0.771725  0.771725  0.771725    0.771725\n","government          0.739130  0.825911  0.780115  247.000000\n","position            0.774554  0.801386  0.787741  433.000000\n","book                0.775000  0.805195  0.789809  154.000000\n","company             0.759124  0.825397  0.790875  378.000000\n","overall_recall      0.797852  0.797852  0.797852    0.797852\n","movie               0.807947  0.807947  0.807947  151.000000\n","game                0.791925  0.864407  0.826580  295.000000\n","name                0.865031  0.909677  0.886792  465.000000\n","overall_accuracy    0.938997  0.938997  0.938997    0.938997"]},"metadata":{},"execution_count":28}]},{"cell_type":"code","source":["#将验证集真实标签和预测结果进行对比展示\n","val_df['preds']=pd.Series(predictions)\n","val_df.to_csv('./bert_lstm_crf/val_1221.csv')\n","val_df=val_df.drop([\"pad_labels\"],axis=1)\n","val_df"],"metadata":{"id":"BnidPIzB9Vjq","colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"status":"ok","timestamp":1640208495567,"user_tz":-480,"elapsed":624,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"0ca00be0-033e-4b80-effe-1c9609d34c53"},"execution_count":31,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","  <div id=\"df-6aa13505-0f0b-488b-b8ff-b87a0df15790\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>words</th>\n","      <th>labels</th>\n","      <th>preds</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>[彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...</td>\n","      <td>[7, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[7, 17, 17, 0, 0, 0, 3, 13, 13, 13, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>[温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...</td>\n","      <td>[7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n","      <td>[7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>[突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...</td>\n","      <td>[4, 14, 14, 14, 14, 14, 14, 14, 0, 7, 17, 17, ...</td>\n","      <td>[6, 16, 16, 16, 16, 16, 16, 16, 0, 7, 17, 17, ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>[郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>[我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...</td>\n","      <td>[0, 0, 0, 0, 10, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1338</th>\n","      <td>[在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n","    </tr>\n","    <tr>\n","      <th>1339</th>\n","      <td>[姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...</td>\n","      <td>[6, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n","      <td>[6, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...</td>\n","    </tr>\n","    <tr>\n","      <th>1340</th>\n","      <td>[目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...</td>\n","      <td>[0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...</td>\n","      <td>[0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>1341</th>\n","      <td>[也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 20, 0, 0, 0, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 20...</td>\n","    </tr>\n","    <tr>\n","      <th>1342</th>\n","      <td>[另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 2, 12, 12, 12, 12, 12, 12, ...</td>\n","      <td>[0, 0, 1, 11, 11, 0, 2, 12, 12, 12, 12, 12, 12...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1343 rows × 3 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6aa13505-0f0b-488b-b8ff-b87a0df15790')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-6aa13505-0f0b-488b-b8ff-b87a0df15790 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-6aa13505-0f0b-488b-b8ff-b87a0df15790');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "],"text/plain":["                                                  words  ...                                              preds\n","0     [彭, 小, 军, 认, 为, ，, 国, 内, 银, 行, 现, 在, 走, 的, 是, ...  ...  [7, 17, 17, 0, 0, 0, 3, 13, 13, 13, 0, 0, 0, 0...\n","1     [温, 格, 的, 球, 队, 终, 于, 又, 踢, 了, 一, 场, 经, 典, 的, ...  ...  [7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...\n","2     [突, 袭, 黑, 暗, 雅, 典, 娜, 》, 中, R, i, d, d, i, c, ...  ...  [6, 16, 16, 16, 16, 16, 16, 16, 0, 7, 17, 17, ...\n","3     [郑, 阿, 姨, 就, 赶, 到, 文, 汇, 路, 排, 队, 拿, 钱, ，, 希, ...  ...  [0, 0, 0, 0, 0, 0, 1, 11, 11, 0, 0, 0, 0, 0, 0...\n","4     [我, 想, 站, 在, 雪, 山, 脚, 下, 你, 会, 被, 那, 巍, 峨, 的, ...  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n","...                                                 ...  ...                                                ...\n","1338  [在, 这, 个, 非, 常, 喜, 庆, 的, 日, 子, 里, ，, 我, 们, 首, ...  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n","1339  [姜, 哲, 中, ：, 公, 共, 之, 敌, 1, -, 1, 》, 、, 《, 神, ...  ...  [6, 17, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16...\n","1340  [目, 前, ，, 日, 本, 松, 山, 海, 上, 保, 安, 部, 正, 在, 就, ...  ...  [0, 0, 0, 5, 15, 15, 15, 15, 15, 15, 15, 15, 0...\n","1341  [也, 就, 是, 说, 英, 国, 人, 在, 世, 博, 会, 上, 的, 英, 国, ...  ...  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 20...\n","1342  [另, 外, 意, 大, 利, 的, P, l, a, y, G, e, n, e, r, ...  ...  [0, 0, 1, 11, 11, 0, 2, 12, 12, 12, 12, 12, 12...\n","\n","[1343 rows x 3 columns]"]},"metadata":{},"execution_count":31}]},{"cell_type":"markdown","metadata":{"id":"pP1vNsUHaalV"},"source":["## 用模型预测验证集结果，与原标签对比"]},{"cell_type":"code","metadata":{"id":"YMQp6C-rZ0S3","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1640208621081,"user_tz":-480,"elapsed":35418,"user":{"displayName":"张hongxu","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"01344108933923387301"}},"outputId":"a9da7377-c8e8-4029-de55-d4e5c3c82846"},"source":["y_pred,y_true,predictions=predict(model,test_loader)\n","pd.DataFrame({'label':predictions}).to_csv('./bert_lstm_crf/submit1222.csv',index=None)"],"execution_count":32,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n","  del sys.path[0]\n"]}]}]}